# -*- coding:utf-8 -*-
from __future__ import unicode_literals
import re
import codecs

from textrank4zh import TextRank4Keyword,TextRank4Sentence

from app.common.mongo_conn import MongoConn
import logging
from newspaper import Article
import json


#关键字提取
class KeywordExtract:

    def __init__(self,  stop_words_file='./app/keyword/stopwords_zh.txt'):
        self.mongoConn = MongoConn()
        self.tableName = 'feeds_temp'
        #basePath =  os.path.abspath('.')
       # stop_words_file = basePath+os.sep+"app"+os.sep+"keyword"+os.sep+"stopwords_zh.txt"

        self.stop_words_file = stop_words_file

    def extract(self, keywordNum=10):
        logging.debug('extract start ...')
        while(True):
            docs = self.mongoConn.coll(self.tableName).find().sort("url",-1).limit(100)
            if docs.count()<1:
                break

            feedsTempIds = []
            for doc in docs:
                feedsTempIds.append(doc["_id"])
                content, title = self.readContent(doc['url'])
                if len(content)<1:
                    continue
                keywords = self.keyWord(content, keywordNum)
                if keywords is not None and len(keywords):
                    self.mongoConn.coll("feeds").update({"_id": doc["feed_id"]},
                                                    {"$set": {"keywords_weight": json.dumps(keywords, ensure_ascii=False)}})

            if len(feedsTempIds)>0:
                self.removeFeedsTemp(feedsTempIds)


    def keyWord(self, content, keywordNum=10):
        tr4w = TextRank4Keyword(stop_words_file=self.stop_words_file)
        tr4w.analyze(text=content, lower=True,
                     window=2)  # py2中text必须是utf8编码的str或者unicode对象，py3中必须是utf8编码的bytes或者str对象
        return tr4w.get_keywords(keywordNum, word_min_len=1)

    def keyWordByUrl(self, url, keywordNum=3):
        content, title = self.readContent(url)
        if content is not None:
            return self.keyWord(content, keywordNum)
        return ''

    def keySentence(self, content, keySentenceNum=3):
        tr4s = TextRank4Sentence(stop_words_file=self.stop_words_file)
        tr4s.analyze(text=content, lower=True, source='all_filters')
        return tr4s.get_key_sentences(num=keySentenceNum)

    def keySentencesByUrl(self, url, keySentenceNum=3):
        content, title = self.readContent(url)
        if content is not None:
            return self.keySentence(content, keySentenceNum)
        return ''

    def filterCodeBlock(self, content):
        #过滤代码块
        result, number = re.subn(r'\u003C\u002Fp\u003E.*|\s*\u003Cp\u003E', '', content,flags=re.IGNORECASE)
        result, number = re.subn(r'\u003Ccode\u003E.*|\s*\u003C\u002Fcode\u003E', '', result,flags=re.IGNORECASE)
        result, number = re.subn(r'\u003Cpre\u003E.*|\s*\u003C\u002Fpre\u003E', '', result,flags=re.IGNORECASE)
        return result

    def readContent(self, url):
        try:
            article = Article(url)
            article.download()
            article.parse()
            text = article.text
            title = article.title
            if text is None or len(text)<1:
                text = title
            return text, title
        except Exception:
            logging.error('read url content failed %s', url)
            return ''

    def removeFeedsTemp(self, ids):
        self.mongoConn.delete(self.tableName, {"_id": {"$in": ids }})

    def keyWordSentencesByUrl(self, url, keywordNum = 3):
        logging.debug("stopwords file %s", self.stop_words_file)
        content, title = self.readContent(url)
        keySentences = self.keySentence(content, 1)
        keyword = self.keyWord(content, keywordNum)
        logging.debug("keyword: %s, keySentences: %s, title: %s", keyword, keySentences, title)
        return title, keyword, keySentences




